<?php
/**
 * Created by PhpStorm.
 * User: 王 朋
 * Date: 2018/5/31
 * Time: 17:10
 */
namespace cauliflower\Library;

class Crawler
{
    /**
     * 从指定URL中获取HTML内容
     * @param $url
     * @return string
     */
    public function _getUrlContent($url) {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        // 设置为true时curl_exec()获取的信息会以字符串返回，而不是直接输出
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        // 启用时会将头文件的信息作为数据流输出
        curl_setopt($ch, CURLOPT_HEADER, 1);
        $content = curl_exec($ch);
        // 最后一个收到的HTTP状态码
        $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        if ($code != '404' && $content) {
            return $content;
        }
        curl_close($ch);
    }

    /**
     * 从HTML页面中筛选需要的内容
     * @param string $content
     * @return array
     */
    public function _filter($content) {
        $pattern = '/<[a|A].*?href=[\'\"]{0,1}([^>\'\"]*).*?>/';
        $result = preg_match_all($pattern, $content, $match_result);
        if ($result) {
            return $match_result[1];
        }
    }

    /**
     * 修正相对路径
     * @param string $base_url
     * @param array $url_list
     * @return array
     */

    public function _reviseUrl ($base_url, $url_list) {
        $url_info = parse_url($base_url);
        $base_url = $url_info['scheme'].'://';
        if ($url_info['user'] && $url_info['pass']) {
            $base_url .= $url_info['user']. ':' . $url_info['pass']. '@';
        }
        $base_url .= $url_info['host'];
        if ($url_info['port']) {
            $base_url .= ':' . $url_info['port'];
        }
        $base_url .= $url_info['path'];
        if (is_array($url_list)) {
            foreach ($url_list as $url_item) {
                if (preg_match('/^http/', $url_item)) {
                    $result[] = $url_item;
                } else {
                    $real_url = $base_url . '/' . $url_item;
                    $result[] = $real_url;
                }
            }
            return $result;
        } else {
            return;
        }
    }

    /**
     * 爬虫
     * @param string $url
     * @return array
     */
    public function _crawler($url) {
        $content = $this->_getUrlContent($url);
        if ($content) {
            $url_list = $this->_reviseUrl($url, $this->_filter($content));
            if ($url_list) {
                return $url_list;
            } else {
                return;
            }
        } else {
            return;
        }
    }

    /**
     * 测试方法
     */
    public function main () {
        $file_path = "./url-01.txt";
        if (file_exists($file_path)) {
            unlink($file_path);
        }
        $current_url = 'https://www.baidu.com';
        $fp_puts = fopen($file_path, 'ab');
        $fp_gets = fopen($file_path, 'r');
        do {
            $url_list = $this->_crawler($current_url);
            echo "<p>$current_url</p>";
            if ($url_list) {
                foreach ($url_list as $url) {
                    fputs($fp_puts, $url . '\r\n');
                }
            }
        } while ($current_url = fgets($fp_gets, 1024));
    }
}

$crawler = new Crawler();
$crawler->main();